<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><title>一文彻底搞懂Flume | 全栈进阶那些事</title><meta name="keywords" content="Hadoop,Flume"><meta name="author" content="liuyuantao"><meta name="copyright" content="liuyuantao"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="本文是基于CentOS 7.9系统环境，进行Flume的学习和使用 一、Flume的简介1.1 Flume基本概念(1) 什么是Flume Flume是Cloudera提供的一个高可用的，高可靠的，分布式的海量日志采集、聚合和传输的系统。(2) Flume的目的 Flume最主要的作业就是，实时读取服务器本地磁盘的数据，将数据写入HDFS 1.2 Flume基本组件(1) Flume工作流程 So">
<meta property="og:type" content="article">
<meta property="og:title" content="一文彻底搞懂Flume">
<meta property="og:url" content="https://liuyuantao.gitee.io/2021/09/12/a99eab3d4a13.html">
<meta property="og:site_name" content="全栈进阶那些事">
<meta property="og:description" content="本文是基于CentOS 7.9系统环境，进行Flume的学习和使用 一、Flume的简介1.1 Flume基本概念(1) 什么是Flume Flume是Cloudera提供的一个高可用的，高可靠的，分布式的海量日志采集、聚合和传输的系统。(2) Flume的目的 Flume最主要的作业就是，实时读取服务器本地磁盘的数据，将数据写入HDFS 1.2 Flume基本组件(1) Flume工作流程 So">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://oscimg.oschina.net/oscnet/20200617165336828.png">
<meta property="article:published_time" content="2021-09-12T05:15:11.000Z">
<meta property="article:modified_time" content="2021-11-13T01:06:19.869Z">
<meta property="article:author" content="liuyuantao">
<meta property="article:tag" content="Hadoop">
<meta property="article:tag" content="Flume">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://oscimg.oschina.net/oscnet/20200617165336828.png"><link rel="shortcut icon" href="/img/favicon.png"><link rel="canonical" href="https://liuyuantao.gitee.io/2021/09/12/a99eab3d4a13"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//hm.baidu.com"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.css" media="print" onload="this.media='all'"><script>var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "https://hm.baidu.com/hm.js?d43383137144f76a37f1074576bc674e";
  var s = document.getElementsByTagName("script")[0]; 
  s.parentNode.insertBefore(hm, s);
})();
</script><script>const GLOBAL_CONFIG = { 
  root: '/',
  algolia: undefined,
  localSearch: {"path":"search.xml","languages":{"hits_empty":"找不到您查询的内容：${query}"}},
  translate: {"defaultEncoding":1,"translateDelay":0,"msgToTraditionalChinese":"繁","msgToSimplifiedChinese":"简"},
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":200},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: true,
    post: true
  },
  runtime: '天',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: {"limitCount":500,"languages":{"author":"作者: liuyuantao","link":"链接: ","source":"来源: 全栈进阶那些事","info":"著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。"}},
  lightbox: 'mediumZoom',
  Snackbar: {"chs_to_cht":"你已切换为繁体","cht_to_chs":"你已切换为简体","day_to_night":"你已切换为深色模式","night_to_day":"你已切换为浅色模式","bgLight":"#49b1f5","bgDark":"#2d3035","position":"top-right"},
  source: {
    jQuery: 'https://cdn.jsdelivr.net/npm/jquery@latest/dist/jquery.min.js',
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/js/jquery.justifiedGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/css/justifiedGallery.min.css'
    },
    fancybox: {
      js: 'https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.js',
      css: 'https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.css'
    }
  },
  isPhotoFigcaption: true,
  islazyload: false,
  isanchor: true
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: '一文彻底搞懂Flume',
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2021-11-13 09:06:19'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const fontSizeVal = saveToLocal.get('global-font-size')
    if (fontSizeVal !== undefined) {
      document.documentElement.style.setProperty('--global-font-size', fontSizeVal + 'px')
    }
    
    const detectApple = () => {
      if (GLOBAL_CONFIG_SITE.isHome && /iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><meta name="generator" content="Hexo 5.4.0"><link rel="alternate" href="/atom.xml" title="全栈进阶那些事" type="application/atom+xml">
</head><body><div id="loading-box"><div class="loading-left-bg"></div><div class="loading-right-bg"></div><div class="spinner-box"><div class="configure-border-1"><div class="configure-core"></div></div><div class="configure-border-2"><div class="configure-core"></div></div><div class="loading-word">加载中...</div></div></div><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="/img/zhiyao_avatar.jpg" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="site-data"><div class="data-item is-center"><div class="data-item-link"><a href="/archives/"><div class="headline">文章</div><div class="length-num">41</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/tags/"><div class="headline">标签</div><div class="length-num">14</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/categories/"><div class="headline">分类</div><div class="length-num">4</div></a></div></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="not-top-img" id="page-header"><nav id="nav"><span id="blog_name"><a id="site-name" href="/">全栈进阶那些事</a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav></header><main class="layout" id="content-inner"><div id="post"><div id="post-info"><h1 class="post-title">一文彻底搞懂Flume</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="fa-fw post-meta-icon far fa-calendar-alt"></i><span class="post-meta-label">发表于</span><time datetime="2021-09-12T05:15:11.000Z" title="undefined 2021-09-12 13:15:11">2021-09-12</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-wordcount"><i class="far fa-file-word fa-fw post-meta-icon"></i><span class="post-meta-label">字数总计:</span><span class="word-count">5.9k</span><span class="post-meta-separator">|</span><i class="far fa-clock fa-fw post-meta-icon"></i><span class="post-meta-label">阅读时长:</span><span>28分钟</span></span><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title="一文彻底搞懂Flume"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"></span></span></div></div></div><article class="post-content" id="article-container"><p>本文是基于CentOS 7.9系统环境，进行Flume的学习和使用</p>
<h1 id="一、Flume的简介"><a href="#一、Flume的简介" class="headerlink" title="一、Flume的简介"></a>一、Flume的简介</h1><h2 id="1-1-Flume基本概念"><a href="#1-1-Flume基本概念" class="headerlink" title="1.1 Flume基本概念"></a>1.1 Flume基本概念</h2><p><strong>(1) 什么是Flume</strong></p>
<p>Flume是Cloudera提供的一个高可用的，高可靠的，分布式的海量日志采集、聚合和传输的系统。<br><strong>(2) Flume的目的</strong></p>
<p>Flume最主要的作业就是，实时读取服务器本地磁盘的数据，将数据写入HDFS</p>
<h2 id="1-2-Flume基本组件"><a href="#1-2-Flume基本组件" class="headerlink" title="1.2 Flume基本组件"></a>1.2 Flume基本组件</h2><p><img src="https://oscimg.oschina.net/oscnet/20200617165336828.png"><br><strong>(1) Flume工作流程</strong></p>
<p>Source采集数据并包装成Event，并将Event缓存在Channel中，Sink不断地从Channel获取Event，并解决成数据，最终将数据写入存储或索引系统<br><strong>(2) Agent</strong></p>
<p>Agent是一个JVM进程，它以事件的形式将数据从源头送至目的。<br>Agent主要有3个部分组成，Source、Channel、Sink<br><strong>(3) Source</strong></p>
<p>Source是负责接收数据到Flume Agent的组件，采集数据并包装成Event。Source组件可以处理各种类型、各种格式的日志数据，包括avro、thrift、exec、jms、spooling directory、netcat、sequence generator、syslog、http、legacy<br><strong>(4) Sink</strong></p>
<p>Sink不断地轮询Channel中的事件且批量地移除它们，并将这些事件批量写入到存储或索引系统、或者被发送到另一个Flume Agent。<br>Sink组件目的地包括hdfs、logger、avro、thrift、ipc、file、HBase、solr、自定义<br><strong>(5) Channel</strong></p>
<p>Channel是位于Source和Sink之间的缓冲区。因此，Channel允许Source和Sink运作在不同的速率上。Channel是线程安全的，可以同时处理几个Source的写入操作和几个Sink的读取操作</p>
<p>Flume自带两种Channel：Memory Channel和File Channel</p>
<p>Memory Channel是内存中的队列。Memory Channel在不需要关心数据丢失的情景下适用。如果需要关心数据丢失，那么Memory Channel就不应该使用，因为程序死亡、机器宕机或者重启都会导致数据丢失</p>
<p>File Channel将所有事件写到磁盘。因此在程序关闭或机器宕机的情况下不会丢失数据</p>
<p><strong>(6) Event</strong></p>
<p>传输单元，Flume数据传输的基本单元，以Event的形式将数据从源头送至目的地。Event由Header和Body两部分组成，Header用来存放该event的一些属性，为K-V结构，Body用来存放该条数据，形式为字节数组</p>
<p><img src="https://oscimg.oschina.net/oscnet/20200618084704461.png"></p>
<h1 id="二、Flume的安装和入门案例"><a href="#二、Flume的安装和入门案例" class="headerlink" title="二、Flume的安装和入门案例"></a>二、Flume的安装和入门案例</h1><h2 id="2-1-Flume安装"><a href="#2-1-Flume安装" class="headerlink" title="2.1 Flume安装"></a>2.1 Flume安装</h2><p><strong>(1) Flume压缩包解压</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tar -xzvf apache-flume-1.7.0-bin.tar.gz -C /opt/module/</span><br></pre></td></tr></table></figure>

<p>(2) 修改Flume名称</p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">cd /opt/module/</span><br><span class="line">mv apache-flume-1.7.0-bin flume</span><br></pre></td></tr></table></figure>

<p>(3) 修改Flume配置文件</p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">cd /opt/module/flume/conf</span><br><span class="line">mv flume-env.sh.template flume-env.sh</span><br><span class="line">vi flume-env.sh</span><br><span class="line"># 修改内容如下</span><br><span class="line">export JAVA_HOME=/opt/module/jdk1.8.0_271</span><br></pre></td></tr></table></figure>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">cd /opt/module/flume/conf</span><br><span class="line">vi log4j.properties</span><br><span class="line"># 修改内容如下</span><br><span class="line">flume.log.dir=/opt/module/flume/logs</span><br></pre></td></tr></table></figure>

<h2 id="2-2-Flume案例-监听数据端口"><a href="#2-2-Flume案例-监听数据端口" class="headerlink" title="2.2 Flume案例-监听数据端口"></a>2.2 Flume案例-监听数据端口</h2><p><img src="https://oscimg.oschina.net/oscnet/20200618113347933.png"><br><strong>(1) 安装nc</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">yum install -y nc</span><br></pre></td></tr></table></figure>

<p><strong>(2) 安装net-tools</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">yum install -y net-tools</span><br></pre></td></tr></table></figure>

<p><strong>(3) 检测端口是否被占用</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">netstat -nltp | grep 444444</span><br></pre></td></tr></table></figure>

<p><strong>(4) 启动flume-agent</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">cd /opt/module/flume</span><br><span class="line">bin/flume-ng agent --name a1 --conf conf/ --conf-file job/flume-netcat-logger.conf -Dflume.root.logger=INFO,console</span><br></pre></td></tr></table></figure>

<p><strong>(5) 开启另一个终端，发送消息</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">nc localhost 4444</span><br><span class="line">aaa</span><br></pre></td></tr></table></figure>

<h2 id="2-3-Flume案例-实时监控单个追加文件"><a href="#2-3-Flume案例-实时监控单个追加文件" class="headerlink" title="2.3 Flume案例-实时监控单个追加文件"></a>2.3 Flume案例-实时监控单个追加文件</h2><p><img src="https://img-blog.csdnimg.cn/20200618164021860.png"><br><strong>(1) 拷贝jar包至/opt/module/flume/lib</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">commons-configuration-1.6.jar</span><br><span class="line">hadoop-auth-2.7.2.jar</span><br><span class="line">hadoop-common-2.7.2.jar</span><br><span class="line">hadoop-hdfs-2.7.2.jar</span><br><span class="line">commons-io-2.4.jar</span><br><span class="line">htrace-core-3.1.0-incubating.jar</span><br></pre></td></tr></table></figure>

<p><strong>(2) 创建flume-file-hdfs.conf文件</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">vi flume-file-hdfs.conf</span><br><span class="line"># Name the components on this agent</span><br><span class="line">a2.sources = r2</span><br><span class="line">a2.sinks = k2</span><br><span class="line">a2.channels = c2</span><br><span class="line"></span><br><span class="line"># Describe/configure the source</span><br><span class="line">a2.sources.r2.type = exec</span><br><span class="line">a2.sources.r2.command = tail -F /opt/module/hive/logs/hive.log</span><br><span class="line">a2.sources.r2.shell = /bin/bash -c</span><br><span class="line"></span><br><span class="line"># Describe the sink</span><br><span class="line">a2.sinks.k2.type = hdfs</span><br><span class="line">a2.sinks.k2.hdfs.path = hdfs://hadoop102:9000/flume/%Y%m%d/%H</span><br><span class="line">#上传文件的前缀</span><br><span class="line">a2.sinks.k2.hdfs.filePrefix = logs-</span><br><span class="line">#是否按照时间滚动文件夹</span><br><span class="line">a2.sinks.k2.hdfs.round = true</span><br><span class="line">#多少时间单位创建一个新的文件夹</span><br><span class="line">a2.sinks.k2.hdfs.roundValue = 1</span><br><span class="line">#重新定义时间单位</span><br><span class="line">a2.sinks.k2.hdfs.roundUnit = hour</span><br><span class="line">#是否使用本地时间戳</span><br><span class="line">a2.sinks.k2.hdfs.useLocalTimeStamp = true</span><br><span class="line">#积攒多少个Event才flush到HDFS一次</span><br><span class="line">a2.sinks.k2.hdfs.batchSize = 1000</span><br><span class="line">#设置文件类型，可支持压缩</span><br><span class="line">a2.sinks.k2.hdfs.fileType = DataStream</span><br><span class="line">#多久生成一个新的文件</span><br><span class="line">a2.sinks.k2.hdfs.rollInterval = 60</span><br><span class="line">#设置每个文件的滚动大小</span><br><span class="line">a2.sinks.k2.hdfs.rollSize = 134217700</span><br><span class="line">#文件的滚动与Event数量无关</span><br><span class="line">a2.sinks.k2.hdfs.rollCount = 0</span><br><span class="line"></span><br><span class="line"># Use a channel which buffers events in memory</span><br><span class="line">a2.channels.c2.type = memory</span><br><span class="line">a2.channels.c2.capacity = 1000</span><br><span class="line">a2.channels.c2.transactionCapacity = 100</span><br><span class="line">    </span><br><span class="line"># Bind the source and sink to the channel</span><br><span class="line">a2.sources.r2.channels = c2</span><br><span class="line">a2.sinks.k2.channel = c2</span><br></pre></td></tr></table></figure>

<p><strong>(3) 启动flume-agent</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent -n a2 -c conf/ -f job/flume-file-hdfs.conf</span><br></pre></td></tr></table></figure>

<p><strong>(4) 开启另一个终端，执行hive命令</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">hive</span><br></pre></td></tr></table></figure>

<p>2.4 Flume案例-实时监控目录下多个新文件</p>
<p><img src="https://oscimg.oschina.net/oscnet/20200618164021860.png"><br><strong>(1) 创建flume-dir-hdfs.conf文件</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">vim flume-dir-hdfs.conf</span><br><span class="line"># 添加如下内容</span><br><span class="line">a3.sources = r3</span><br><span class="line">a3.sinks = k3</span><br><span class="line">a3.channels = c3</span><br><span class="line"></span><br><span class="line"># Describe/configure the source</span><br><span class="line">a3.sources.r3.type = spooldir</span><br><span class="line">a3.sources.r3.spoolDir = /opt/module/flume/upload</span><br><span class="line">a3.sources.r3.fileSuffix = .COMPLETED</span><br><span class="line">a3.sources.r3.fileHeader = true</span><br><span class="line">#忽略所有以.tmp结尾的文件，不上传</span><br><span class="line">a3.sources.r3.ignorePattern = ([^ ]*\.tmp)</span><br><span class="line"></span><br><span class="line"># Describe the sink</span><br><span class="line">a3.sinks.k3.type = hdfs</span><br><span class="line">a3.sinks.k3.hdfs.path = hdfs://hadoop102:9000/flume/upload/%Y%m%d/%H</span><br><span class="line">#上传文件的前缀</span><br><span class="line">a3.sinks.k3.hdfs.filePrefix = upload-</span><br><span class="line">#是否按照时间滚动文件夹</span><br><span class="line">a3.sinks.k3.hdfs.round = true</span><br><span class="line">#多少时间单位创建一个新的文件夹</span><br><span class="line">a3.sinks.k3.hdfs.roundValue = 1</span><br><span class="line">#重新定义时间单位</span><br><span class="line">a3.sinks.k3.hdfs.roundUnit = hour</span><br><span class="line">#是否使用本地时间戳</span><br><span class="line">a3.sinks.k3.hdfs.useLocalTimeStamp = true</span><br><span class="line">#积攒多少个Event才flush到HDFS一次</span><br><span class="line">a3.sinks.k3.hdfs.batchSize = 100</span><br><span class="line">#设置文件类型，可支持压缩</span><br><span class="line">a3.sinks.k3.hdfs.fileType = DataStream</span><br><span class="line">#多久生成一个新的文件</span><br><span class="line">a3.sinks.k3.hdfs.rollInterval = 60</span><br><span class="line">#设置每个文件的滚动大小大概是128M</span><br><span class="line">a3.sinks.k3.hdfs.rollSize = 134217700</span><br><span class="line">#文件的滚动与Event数量无关</span><br><span class="line">a3.sinks.k3.hdfs.rollCount = 0</span><br><span class="line"></span><br><span class="line"># Use a channel which buffers events in memory</span><br><span class="line">a3.channels.c3.type = memory</span><br><span class="line">a3.channels.c3.capacity = 1000</span><br><span class="line">a3.channels.c3.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"># Bind the source and sink to the channel</span><br><span class="line">a3.sources.r3.channels = c3</span><br><span class="line">a3.sinks.k3.channel = c3</span><br></pre></td></tr></table></figure>

<p><strong>(2) 启动flume-agent</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent -n a3 -c conf/ -f job/flume-dir-hdfs.conf</span><br></pre></td></tr></table></figure>

<p><strong>(3) 开启另一个终端</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">cd /opt/module/flume/</span><br><span class="line">mkdir upload</span><br><span class="line">cp NOTICE upload/</span><br></pre></td></tr></table></figure>

<h2 id="2-5-Flume案例-实时监控目录下的多个追加文件"><a href="#2-5-Flume案例-实时监控目录下的多个追加文件" class="headerlink" title="2.5 Flume案例-实时监控目录下的多个追加文件"></a>2.5 Flume案例-实时监控目录下的多个追加文件</h2><p>Exec source适用于监控一个实时追加的文件，不能实现断电续传；Spooldir Source适合用于同步新文件，但不适合对实时追加日志的文件进行监听并同步；而Taildir Source适合用于监听多个实时追加的文件，并且能够实现断点续传。</p>
<p><img src="https://oscimg.oschina.net/oscnet/20200618164251157.png"><br><strong>(1) 创建flume-dir-hdfs.conf文件</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi flume-taildir-hdfs.conf</span><br><span class="line"><span class="comment"># 添加内容</span></span><br><span class="line">a3.sources = r3</span><br><span class="line">a3.sinks = k3</span><br><span class="line">a3.channels = c3</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a3.sources.r3.type = TAILDIR</span><br><span class="line">a3.sources.r3.positionFile = /opt/module/flume/tail_dir.json</span><br><span class="line">a3.sources.r3.filegroups = f1 f2</span><br><span class="line">a3.sources.r3.filegroups.f1 = /opt/module/flume/files/.*file.*</span><br><span class="line">a3.sources.r3.filegroups.f2 = /opt/module/flume/files/.*<span class="built_in">log</span>.*</span><br><span class="line"><span class="comment"># 不能写成如下的形式，后面的会覆盖前面的</span></span><br><span class="line"><span class="comment"># a3.sources.r3.filegroups.f1 = /opt/module/flume/files/file1.txt</span></span><br><span class="line"><span class="comment"># a3.sources.r3.filegroups.f1 = /opt/module/flume/files/file2.txt</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a3.sinks.k3.type = hdfs</span><br><span class="line">a3.sinks.k3.hdfs.path = hdfs://hadoop102:9000/flume/upload2/%Y%m%d/%H</span><br><span class="line"><span class="comment">#上传文件的前缀</span></span><br><span class="line">a3.sinks.k3.hdfs.filePrefix = upload-</span><br><span class="line"><span class="comment">#是否按照时间滚动文件夹</span></span><br><span class="line">a3.sinks.k3.hdfs.round = <span class="literal">true</span></span><br><span class="line"><span class="comment">#多少时间单位创建一个新的文件夹</span></span><br><span class="line">a3.sinks.k3.hdfs.roundValue = 1</span><br><span class="line"><span class="comment">#重新定义时间单位</span></span><br><span class="line">a3.sinks.k3.hdfs.roundUnit = hour</span><br><span class="line"><span class="comment">#是否使用本地时间戳</span></span><br><span class="line">a3.sinks.k3.hdfs.useLocalTimeStamp = <span class="literal">true</span></span><br><span class="line"><span class="comment">#积攒多少个Event才flush到HDFS一次</span></span><br><span class="line">a3.sinks.k3.hdfs.batchSize = 100</span><br><span class="line"><span class="comment">#设置文件类型，可支持压缩</span></span><br><span class="line">a3.sinks.k3.hdfs.fileType = DataStream</span><br><span class="line"><span class="comment">#多久生成一个新的文件</span></span><br><span class="line">a3.sinks.k3.hdfs.rollInterval = 60</span><br><span class="line"><span class="comment">#设置每个文件的滚动大小大概是128M</span></span><br><span class="line">a3.sinks.k3.hdfs.rollSize = 134217700</span><br><span class="line"><span class="comment">#文件的滚动与Event数量无关</span></span><br><span class="line">a3.sinks.k3.hdfs.rollCount = 0</span><br><span class="line"></span><br><span class="line"><span class="comment"># Use a channel which buffers events in memory</span></span><br><span class="line">a3.channels.c3.type = memory</span><br><span class="line">a3.channels.c3.capacity = 1000</span><br><span class="line">a3.channels.c3.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a3.sources.r3.channels = c3</span><br><span class="line">a3.sinks.k3.channel = c3</span><br></pre></td></tr></table></figure>

<p><strong>(2) 创建目录和文件</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">cd /opt/module/flume</span><br><span class="line">mkdir files</span><br><span class="line">cp CHANGELOG files/CHANGELOG.log</span><br><span class="line">cp LICENSE files/LICENSE.log</span><br></pre></td></tr></table></figure>

<p><strong>(3) 启动flume-agent</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent -n a3 -c conf/ -f job/flume-taildir-hdfs.conf</span><br></pre></td></tr></table></figure>

<p>(4) 开启另一个终端</p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">cd /opt/module/flume/files</span><br><span class="line">vi CHANGELOG.log</span><br><span class="line"># 添加如下内容</span><br><span class="line">xxxxx</span><br><span class="line">sssss</span><br><span class="line">wwwww</span><br></pre></td></tr></table></figure>

<h1 id="三、Flume的进阶"><a href="#三、Flume的进阶" class="headerlink" title="三、Flume的进阶"></a>三、Flume的进阶</h1><h2 id="3-1-Flume事务"><a href="#3-1-Flume事务" class="headerlink" title="3.1 Flume事务"></a>3.1 Flume事务</h2><p><img src="https://oscimg.oschina.net/oscnet/20200618193742769.png"><br><strong>(1) Put事务流程</strong></p>
<p>doPut：将批数据先写入临时缓存区putList<br>    doCommit：检查channel内存队列是否足够合并<br>    doRollback：channel内存队列空间不足，回滚数据</p>
<p><strong>(2) Take事务流程</strong></p>
<p>doTake：将数据取到临时缓存区takeList，并将数据发送到HDFS<br>    doCommit：如果数据全部发送成功，则清除临时缓冲区takeList<br>    doRollback：数据发送过程中如果出现异常，rollback将临时缓冲区takeList中的数据归还给channel内存队列</p>
<h2 id="3-2-Flume-Agent内部原理"><a href="#3-2-Flume-Agent内部原理" class="headerlink" title="3.2 Flume Agent内部原理"></a>3.2 Flume Agent内部原理</h2><p><img src="https://oscimg.oschina.net/oscnet/20200618193807677.png"><br><strong>(1) ChannelSelector</strong></p>
<p>ChannelSelector的作用就是选出Event将要被发往哪个Channel，其共有两种类型</p>
<p>Replicating（复制）<br>    ReplicatingSelector会将同一个Event发往所有的Channel，<br>    和Multiplexing（多路复用）<br>    Multiplexing会根据相应的原则，将不同的Event发往不同的Channel</p>
<p><strong>(2) SinkProcessor</strong></p>
<p>SinkProcessor共有三种类型</p>
<p>DefaultSinkProcessor<br>    对应单个sink，发送至单个sink<br>    LoadBalancingSinkProcessor<br>    LoadBalancingSinkProcessor对应的是Sink Group，LoadBalancingSinkProcessor可以实现负载均衡的功能<br>    FailoverSinkProcessor<br>    FailoverSinkProcessor对应的是Sink Group，<br>    FailoverSinkProcessor可以错误恢复的功能</p>
<h1 id="四、Flume的拓扑结构"><a href="#四、Flume的拓扑结构" class="headerlink" title="四、Flume的拓扑结构"></a>四、Flume的拓扑结构</h1><p>4.1 简单串联</p>
<p><img src="https://oscimg.oschina.net/oscnet/20200618212349829.png"><br>这种模式是将多个flume顺序连接起来了，从最初的source开始到最终sink传送的目的存储系统。</p>
<p><strong>优点</strong><br>    多个flume并联，可以增加event缓存数量<br><strong>缺点</strong><br>    此模式不建议桥接过多的flume数量， flume数量过多不仅会影响传输速率，而且一旦传输过程中某个节点flume宕机，会影响整个传输系统。</p>
<h2 id="4-2-复制和多路复用"><a href="#4-2-复制和多路复用" class="headerlink" title="4.2 复制和多路复用"></a>4.2 复制和多路复用</h2><p><img src="https://oscimg.oschina.net/oscnet/20200618214258691.png"><br>Flume支持将事件流向一个或者多个目的地。这种模式可以将相同数据复制到多个channel中，或者将不同数据分发到不同的channel中，sink可以选择传送到不同的目的地。</p>
<h2 id="4-3-负载均衡和故障转移"><a href="#4-3-负载均衡和故障转移" class="headerlink" title="4.3 负载均衡和故障转移"></a>4.3 负载均衡和故障转移</h2><p><img src="https://oscimg.oschina.net/oscnet/20200618220201601.png"><br>Flume支持使用将多个sink逻辑上分到一个sink组，sink组配合不同的SinkProcessor可以实现负载均衡和错误恢复的功能。</p>
<h2 id="4-4-聚合"><a href="#4-4-聚合" class="headerlink" title="4.4 聚合"></a>4.4 聚合</h2><p><img src="https://oscimg.oschina.net/oscnet/20200618220241209.png"><br>这种模式是我们最常见的，也非常实用，日常web应用通常分布在上百个服务器，大者甚至上千个、上万个服务器。产生的日志，处理起来也非常麻烦。用flume的这种组合方式能很好的解决这一问题，每台服务器部署一个flume采集日志，传送到一个集中收集日志的flume，再由此flume上传到hdfs、hive、hbase等，进行日志分析。</p>
<h1 id="五、Flume的企业开发实例"><a href="#五、Flume的企业开发实例" class="headerlink" title="五、Flume的企业开发实例"></a>五、Flume的企业开发实例</h1><h2 id="5-1-复制和多路复用"><a href="#5-1-复制和多路复用" class="headerlink" title="5.1 复制和多路复用"></a>5.1 复制和多路复用</h2><p><img src="https://oscimg.oschina.net/oscnet/20200619100122518.png"><br><strong>(1) 创建flume-file-avro.conf文件</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi flume-file-avro.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a1.sources = r1</span><br><span class="line">a1.sinks = k1 k2</span><br><span class="line">a1.channels = c1 c2</span><br><span class="line"><span class="comment"># 将数据流复制给所有channel</span></span><br><span class="line">a1.sources.r1.selector.type = replicating</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a1.sources.r1.type = <span class="built_in">exec</span></span><br><span class="line">a1.sources.r1.command = tail -F /opt/module/hive/logs/hive.log</span><br><span class="line">a1.sources.r1.shell = /bin/bash -c</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line"><span class="comment"># sink端的avro是一个数据发送者</span></span><br><span class="line">a1.sinks.k1.type = avro</span><br><span class="line">a1.sinks.k1.hostname = hadoop1021</span><br><span class="line">a1.sinks.k1.port = 4141</span><br><span class="line"></span><br><span class="line">a1.sinks.k2.type = avro</span><br><span class="line">a1.sinks.k2.hostname = hadoop101</span><br><span class="line">a1.sinks.k2.port = 4142</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the channel</span></span><br><span class="line">a1.channels.c1.type = memory</span><br><span class="line">a1.channels.c1.capacity = 1000</span><br><span class="line">a1.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line">a1.channels.c2.type = memory</span><br><span class="line">a1.channels.c2.capacity = 1000</span><br><span class="line">a1.channels.c2.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a1.sources.r1.channels = c1 c2</span><br><span class="line">a1.sinks.k1.channel = c1</span><br><span class="line">a1.sinks.k2.channel = c2</span><br></pre></td></tr></table></figure>

<p><strong>(2) 创建flume-avro-hdfs.conf文件</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi flume-avro-hdfs.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a2.sources = r1</span><br><span class="line">a2.sinks = k1</span><br><span class="line">a2.channels = c1</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line"><span class="comment"># source端的avro是一个数据接收服务</span></span><br><span class="line">a2.sources.r1.type = avro</span><br><span class="line">a2.sources.r1.bind = hadoop101</span><br><span class="line">a2.sources.r1.port = 4141</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a2.sinks.k1.type = hdfs</span><br><span class="line">a2.sinks.k1.hdfs.path = hdfs://hadoop102:9000/flume2/%Y%m%d/%H</span><br><span class="line"><span class="comment">#上传文件的前缀</span></span><br><span class="line">a2.sinks.k1.hdfs.filePrefix = flume2-</span><br><span class="line"><span class="comment">#是否按照时间滚动文件夹</span></span><br><span class="line">a2.sinks.k1.hdfs.round = <span class="literal">true</span></span><br><span class="line"><span class="comment">#多少时间单位创建一个新的文件夹</span></span><br><span class="line">a2.sinks.k1.hdfs.roundValue = 1</span><br><span class="line"><span class="comment">#重新定义时间单位</span></span><br><span class="line">a2.sinks.k1.hdfs.roundUnit = hour</span><br><span class="line"><span class="comment">#是否使用本地时间戳</span></span><br><span class="line">a2.sinks.k1.hdfs.useLocalTimeStamp = <span class="literal">true</span></span><br><span class="line"><span class="comment">#积攒多少个Event才flush到HDFS一次</span></span><br><span class="line">a2.sinks.k1.hdfs.batchSize = 100</span><br><span class="line"><span class="comment">#设置文件类型，可支持压缩</span></span><br><span class="line">a2.sinks.k1.hdfs.fileType = DataStream</span><br><span class="line"><span class="comment">#多久生成一个新的文件</span></span><br><span class="line">a2.sinks.k1.hdfs.rollInterval = 600</span><br><span class="line"><span class="comment">#设置每个文件的滚动大小大概是128M</span></span><br><span class="line">a2.sinks.k1.hdfs.rollSize = 134217700</span><br><span class="line"><span class="comment">#文件的滚动与Event数量无关</span></span><br><span class="line">a2.sinks.k1.hdfs.rollCount = 0</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the channel</span></span><br><span class="line">a2.channels.c1.type = memory</span><br><span class="line">a2.channels.c1.capacity = 1000</span><br><span class="line">a2.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a2.sources.r1.channels = c1</span><br><span class="line">a2.sinks.k1.channel = c1</span><br></pre></td></tr></table></figure>

<p><strong>(3) 创建flume-avro-dir.conf文件</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi flume-avro-dir.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a3.sources = r1</span><br><span class="line">a3.sinks = k1</span><br><span class="line">a3.channels = c2</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a3.sources.r1.type = avro</span><br><span class="line">a3.sources.r1.bind = hadoop101</span><br><span class="line">a3.sources.r1.port = 4142</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a3.sinks.k1.type = file_roll</span><br><span class="line">a3.sinks.k1.sink.directory = /opt/module/flume/data/flume3</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the channel</span></span><br><span class="line">a3.channels.c2.type = memory</span><br><span class="line">a3.channels.c2.capacity = 1000</span><br><span class="line">a3.channels.c2.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a3.sources.r1.channels = c2</span><br><span class="line">a3.sinks.k1.channel = c2</span><br></pre></td></tr></table></figure>

<p><strong>(4) 执行配置文件</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent -n a3 -c conf/ -f job/group1/flume-avro-dir.conf</span><br><span class="line">bin/flume-ng agent -n a2 -c conf/ -f job/group1/flume-avro-hdfs.conf</span><br><span class="line">bin/flume-ng agent -n a1 -c conf/ -f job/group1/flume-file-avro.conf```</span><br><span class="line"></span><br><span class="line">**(5) 启动Hadoop和Hive**</span><br><span class="line">```bash</span><br><span class="line">sbin/start-dfs.sh</span><br><span class="line">sbin/start-yarn.sh</span><br><span class="line">bin/hive</span><br></pre></td></tr></table></figure>

<h2 id="5-2-故障转移"><a href="#5-2-故障转移" class="headerlink" title="5.2 故障转移"></a>5.2 故障转移</h2><p><img src="https://oscimg.oschina.net/oscnet/20200619105749441.png"><br><strong>(1) 创建a1.conf文件</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi a1.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a1.sources = r1</span><br><span class="line">a1.channels = c1</span><br><span class="line">a1.sinks = k1 k2</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a1.sources.r1.type = netcat</span><br><span class="line">a1.sources.r1.bind = localhost</span><br><span class="line">a1.sources.r1.port = 44444</span><br><span class="line"></span><br><span class="line">a1.sinkgroups = g1</span><br><span class="line">a1.sinkgroups.g1.sinks = k1 k2</span><br><span class="line">a1.sinkgroups.g1.processor.type = failover</span><br><span class="line">a1.sinkgroups.g1.processor.priority.k1 = 5</span><br><span class="line">a1.sinkgroups.g1.processor.priority.k2 = 10</span><br><span class="line">a1.sinkgroups.g1.processor.maxpenalty = 10000</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a1.sinks.k1.type = avro</span><br><span class="line">a1.sinks.k1.hostname = hadoop101</span><br><span class="line">a1.sinks.k1.port = 4141</span><br><span class="line"></span><br><span class="line">a1.sinks.k2.type = avro</span><br><span class="line">a1.sinks.k2.hostname = hadoop101</span><br><span class="line">a1.sinks.k2.port = 4142</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the channel</span></span><br><span class="line">a1.channels.c1.type = memory</span><br><span class="line">a1.channels.c1.capacity = 1000</span><br><span class="line">a1.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a1.sources.r1.channels = c1</span><br><span class="line">a1.sinks.k1.channel = c1</span><br><span class="line">a1.sinks.k2.channel = c1</span><br></pre></td></tr></table></figure>

<p><strong>(2) 创建a2.conf文件</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi a2.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a2.sources = r1</span><br><span class="line">a2.sinks = k1</span><br><span class="line">a2.channels = c1</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a2.sources.r1.type = avro</span><br><span class="line">a2.sources.r1.bind = hadoop101</span><br><span class="line">a2.sources.r1.port = 4141</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a2.sinks.k1.type = logger</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the channel</span></span><br><span class="line">a2.channels.c1.type = memory</span><br><span class="line">a2.channels.c1.capacity = 1000</span><br><span class="line">a2.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a2.sources.r1.channels = c1</span><br><span class="line">a2.sinks.k1.channel = c1</span><br></pre></td></tr></table></figure>

<p><strong>(3) 创建a3.conf文件</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi a3.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a3.sources = r1</span><br><span class="line">a3.sinks = k1</span><br><span class="line">a3.channels = c2</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a3.sources.r1.type = avro</span><br><span class="line">a3.sources.r1.bind = hadoop101</span><br><span class="line">a3.sources.r1.port = 4142</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a3.sinks.k1.type = logger</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the channel</span></span><br><span class="line">a3.channels.c2.type = memory</span><br><span class="line">a3.channels.c2.capacity = 1000</span><br><span class="line">a3.channels.c2.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a3.sources.r1.channels = c2</span><br><span class="line">a3.sinks.k1.channel = c2</span><br></pre></td></tr></table></figure>

<p><strong>(4) 执行配置文件</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent -n a3 -c conf/ -f job/group2/a3.conf -Dflume.root.logger=INFO,console</span><br><span class="line">bin/flume-ng agent -n a2 -c conf/ -f job/group2/a2.conf -Dflume.root.logger=INFO,console</span><br><span class="line">bin/flume-ng agent -n a1 -c conf/ -f job/group2/a1.conf</span><br></pre></td></tr></table></figure>

<p><strong>(5) 开启另一个终端，发送消息</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">nc localhost 4444</span><br><span class="line">aaa</span><br></pre></td></tr></table></figure>

<p><strong>(6) 杀死a3后，通过故障转移，a2能正常工作</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">kill</span> -9 a3-pid</span><br></pre></td></tr></table></figure>

<h2 id="5-3-负载均衡"><a href="#5-3-负载均衡" class="headerlink" title="5.3 负载均衡"></a>5.3 负载均衡</h2><p><img src="https://oscimg.oschina.net/oscnet/20200619111514941.png"><br><strong>(1) 创建a1.conf文件</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi a1.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a1.sources = r1</span><br><span class="line">a1.channels = c1</span><br><span class="line">a1.sinks = k1 k2</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a1.sources.r1.type = netcat</span><br><span class="line">a1.sources.r1.bind = localhost</span><br><span class="line">a1.sources.r1.port = 44444</span><br><span class="line"></span><br><span class="line">a1.sinkgroups = g1</span><br><span class="line">a1.sinkgroups.g1.sinks = k1 k2</span><br><span class="line">a1.sinkgroups.g1.processor.type = load_balance</span><br><span class="line">a1.sinkgroups.g1.processor.backoff = <span class="literal">true</span></span><br><span class="line">a1.sinkgroups.g1.processor.selector = random</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a1.sinks.k1.type = avro</span><br><span class="line">a1.sinks.k1.hostname = hadoop101</span><br><span class="line">a1.sinks.k1.port = 4141</span><br><span class="line"></span><br><span class="line">a1.sinks.k2.type = avro</span><br><span class="line">a1.sinks.k2.hostname = hadoop101</span><br><span class="line">a1.sinks.k2.port = 4142</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the channel</span></span><br><span class="line">a1.channels.c1.type = memory</span><br><span class="line">a1.channels.c1.capacity = 1000</span><br><span class="line">a1.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a1.sources.r1.channels = c1</span><br><span class="line">a1.sinks.k1.channel = c1</span><br><span class="line">a1.sinks.k2.channel = c1</span><br></pre></td></tr></table></figure>

<p><strong>(2) 创建a2.conf文件</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi a2.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a2.sources = r1</span><br><span class="line">a2.sinks = k1</span><br><span class="line">a2.channels = c1</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a2.sources.r1.type = avro</span><br><span class="line">a2.sources.r1.bind = hadoop101</span><br><span class="line">a2.sources.r1.port = 4141</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a2.sinks.k1.type = logger</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the channel</span></span><br><span class="line">a2.channels.c1.type = memory</span><br><span class="line">a2.channels.c1.capacity = 1000</span><br><span class="line">a2.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a2.sources.r1.channels = c1</span><br><span class="line">a2.sinks.k1.channel = c1</span><br></pre></td></tr></table></figure>

<p>(3) 创建a3.conf文件</p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi a3.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a3.sources = r1</span><br><span class="line">a3.sinks = k1</span><br><span class="line">a3.channels = c2</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a3.sources.r1.type = avro</span><br><span class="line">a3.sources.r1.bind = hadoop101</span><br><span class="line">a3.sources.r1.port = 4142</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a3.sinks.k1.type = logger</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the channel</span></span><br><span class="line">a3.channels.c2.type = memory</span><br><span class="line">a3.channels.c2.capacity = 1000</span><br><span class="line">a3.channels.c2.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a3.sources.r1.channels = c2</span><br><span class="line">a3.sinks.k1.channel = c2</span><br></pre></td></tr></table></figure>

<p><strong>(4) 执行配置文件</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent -n a3 -c conf/ -f job/group2/a3.conf -Dflume.root.logger=INFO,console</span><br><span class="line">bin/flume-ng agent -n a2 -c conf/ -f job/group2/a2.conf -Dflume.root.logger=INFO,console</span><br><span class="line">bin/flume-ng agent -n a1 -c conf/ -f job/group2/a1.conf</span><br></pre></td></tr></table></figure>

<p><strong>(5) 开启另一个终端，不断发送消息</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">nc localhost 4444</span><br><span class="line">aaa</span><br></pre></td></tr></table></figure>

<h2 id="5-4-聚合"><a href="#5-4-聚合" class="headerlink" title="5.4 聚合"></a>5.4 聚合</h2><p><img src="https://oscimg.oschina.net/oscnet/20200619131332422.png"><br><strong>(1) 创建a1.conf文件</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi a1.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a1.sources = r1</span><br><span class="line">a1.sinks = k1</span><br><span class="line">a1.channels = c1</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a1.sources.r1.type = <span class="built_in">exec</span></span><br><span class="line">a1.sources.r1.command = tail -F /opt/module/flume/group.log</span><br><span class="line">a1.sources.r1.shell = /bin/bash -c</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a1.sinks.k1.type = avro</span><br><span class="line">a1.sinks.k1.hostname = hadoop103</span><br><span class="line">a1.sinks.k1.port = 4141</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the channel</span></span><br><span class="line">a1.channels.c1.type = memory</span><br><span class="line">a1.channels.c1.capacity = 1000</span><br><span class="line">a1.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a1.sources.r1.channels = c1</span><br><span class="line">a1.sinks.k1.channel = c1</span><br></pre></td></tr></table></figure>

<p><strong>(2) 创建a2.conf文件</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi a2.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a2.sources = r1</span><br><span class="line">a2.sinks = k1</span><br><span class="line">a2.channels = c1</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a2.sources.r1.type = netcat</span><br><span class="line">a2.sources.r1.bind = hadoop102</span><br><span class="line">a2.sources.r1.port = 44444</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a2.sinks.k1.type = avro</span><br><span class="line">a2.sinks.k1.hostname = hadoop103</span><br><span class="line">a2.sinks.k1.port = 4141</span><br><span class="line"></span><br><span class="line"><span class="comment"># Use a channel which buffers events in memory</span></span><br><span class="line">a2.channels.c1.type = memory</span><br><span class="line">a2.channels.c1.capacity = 1000</span><br><span class="line">a2.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a2.sources.r1.channels = c1</span><br><span class="line">a2.sinks.k1.channel = c1</span><br></pre></td></tr></table></figure>

<p><strong>(3) 创建a3.conf文件</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi a3.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a3.sources = r1</span><br><span class="line">a3.sinks = k1</span><br><span class="line">a3.channels = c1</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a3.sources.r1.type = avro</span><br><span class="line">a3.sources.r1.bind = hadoop103</span><br><span class="line">a3.sources.r1.port = 4141</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a3.sinks.k1.type = logger</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the channel</span></span><br><span class="line">a3.channels.c1.type = memory</span><br><span class="line">a3.channels.c1.capacity = 1000</span><br><span class="line">a3.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a3.sources.r1.channels = c1</span><br><span class="line">a3.sinks.k1.channel = c1</span><br></pre></td></tr></table></figure>

<p><strong>(4) 执行配置文件</strong><br><strong>hadoop103</strong></p>
<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent --conf conf/ --name a3 --conf-file job/group4/a3.conf -Dflume.root.logger=INFO,console</span><br></pre></td></tr></table></figure>

<p><strong>hadoop102</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent --conf conf/ --name a2 --conf-file job/group4/a2.conf</span><br></pre></td></tr></table></figure>

<p><strong>hadoop101</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent --conf conf/ --name a1 --conf-file job/group4/a1.conf</span><br></pre></td></tr></table></figure>

<p><strong>(5) 开启另一个终端，不断发送消息</strong></p>
<p><strong>hadoop101</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">nc hadoop102 44444</span><br><span class="line">aaa</span><br></pre></td></tr></table></figure>

<p><strong>(6) 向group.log文件中，添加内容</strong></p>
<p><strong>hadoop101</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">cd</span> /opt/module/flume</span><br><span class="line"><span class="built_in">echo</span> 222 &gt;&gt; group.log</span><br></pre></td></tr></table></figure>

<h2 id="5-5-自定义Interceptor案例"><a href="#5-5-自定义Interceptor案例" class="headerlink" title="5.5 自定义Interceptor案例"></a>5.5 自定义Interceptor案例</h2><p>根据日志不同的类型（type），将日志进行分流，分入到不同的sink<br><strong>(1) 实现一个Interceptor接口</strong></p>
<figure class="highlight java"><table><tr><td class="code"><pre><span class="line"><span class="keyword">package</span> com.lytdevflume.interceptor;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> org.apache.flume.Context;</span><br><span class="line"><span class="keyword">import</span> org.apache.flume.Event;</span><br><span class="line"><span class="keyword">import</span> org.apache.flume.interceptor.Interceptor;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> java.util.List;</span><br><span class="line"><span class="keyword">import</span> java.util.Map;</span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">MyInterceptor</span> <span class="keyword">implements</span> <span class="title">Interceptor</span> </span>&#123;</span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">initialize</span><span class="params">()</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> Event <span class="title">intercept</span><span class="params">(Event event)</span> </span>&#123;</span><br><span class="line">        Map&lt;String, String&gt; headers = event.getHeaders();</span><br><span class="line">        <span class="keyword">byte</span>[] body = event.getBody();</span><br><span class="line">        <span class="keyword">if</span> (body[<span class="number">0</span>] &lt;= <span class="string">&#x27;9&#x27;</span> &amp;&amp; body[<span class="number">0</span>] &gt;= <span class="string">&#x27;0&#x27;</span>) &#123;</span><br><span class="line">            headers.put(<span class="string">&quot;type&quot;</span>, <span class="string">&quot;number&quot;</span>);</span><br><span class="line">        &#125; <span class="keyword">else</span> &#123;</span><br><span class="line">            headers.put(<span class="string">&quot;type&quot;</span>, <span class="string">&quot;not_number&quot;</span>);</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">return</span> event;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> List&lt;Event&gt; <span class="title">intercept</span><span class="params">(List&lt;Event&gt; events)</span> </span>&#123;</span><br><span class="line">        <span class="keyword">for</span> (Event event : events) &#123;</span><br><span class="line">            intercept(event);</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">return</span> events;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">close</span><span class="params">()</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">public</span> <span class="keyword">static</span> <span class="class"><span class="keyword">class</span> <span class="title">MyBuilder</span> <span class="keyword">implements</span> <span class="title">Interceptor</span>.<span class="title">Builder</span></span>&#123;</span><br><span class="line">        <span class="function"><span class="keyword">public</span> Interceptor <span class="title">build</span><span class="params">()</span> </span>&#123;</span><br><span class="line">            <span class="keyword">return</span> <span class="keyword">new</span> MyInterceptor();</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">configure</span><span class="params">(Context context)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<p><strong>(2) hadoop101创建配置文件a1.conf</strong></p>
<p><strong>hadoop101</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">cd</span> /opt/module/flume/job/interceptor</span><br><span class="line">vi a1.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a1.sources = r1</span><br><span class="line">a1.sinks = k1 k2</span><br><span class="line">a1.channels = c1 c2</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a1.sources.r1.type = netcat</span><br><span class="line">a1.sources.r1.bind = localhost</span><br><span class="line">a1.sources.r1.port = 44444</span><br><span class="line">a1.sources.r1.interceptors = i1</span><br><span class="line">a1.sources.r1.interceptors.i1.type = com.lytdevflume.interceptor.MyInterceptor<span class="variable">$MyBuilder</span></span><br><span class="line">a1.sources.r1.selector.type = multiplexing</span><br><span class="line">a1.sources.r1.selector.header = <span class="built_in">type</span></span><br><span class="line">a1.sources.r1.selector.mapping.not_number = c1</span><br><span class="line">a1.sources.r1.selector.mapping.number = c2</span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a1.sinks.k1.type = avro</span><br><span class="line">a1.sinks.k1.hostname = hadoop102</span><br><span class="line">a1.sinks.k1.port = 4141</span><br><span class="line"></span><br><span class="line">a1.sinks.k2.type=avro</span><br><span class="line">a1.sinks.k2.hostname = hadoop103</span><br><span class="line">a1.sinks.k2.port = 4242</span><br><span class="line"></span><br><span class="line"><span class="comment"># Use a channel which buffers events in memory</span></span><br><span class="line">a1.channels.c1.type = memory</span><br><span class="line">a1.channels.c1.capacity = 1000</span><br><span class="line">a1.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Use a channel which buffers events in memory</span></span><br><span class="line">a1.channels.c2.type = memory</span><br><span class="line">a1.channels.c2.capacity = 1000</span><br><span class="line">a1.channels.c2.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a1.sources.r1.channels = c1 c2</span><br><span class="line">a1.sinks.k1.channel = c1</span><br><span class="line">a1.sinks.k2.channel = c2</span><br></pre></td></tr></table></figure>

<p><strong>(3) hadoop102创建配置文件a1.conf</strong></p>
<p><strong>hadoop102</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">cd</span> /opt/module/flume/job/interceptor</span><br><span class="line">vi a1.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a1.sources = r1</span><br><span class="line">a1.sinks = k1</span><br><span class="line">a1.channels = c1</span><br><span class="line"></span><br><span class="line">a1.sources.r1.type = avro</span><br><span class="line">a1.sources.r1.bind = hadoop102</span><br><span class="line">a1.sources.r1.port = 4141</span><br><span class="line"></span><br><span class="line">a1.sinks.k1.type = logger</span><br><span class="line"></span><br><span class="line">a1.channels.c1.type = memory</span><br><span class="line">a1.channels.c1.capacity = 1000</span><br><span class="line">a1.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line">a1.sinks.k1.channel = c1</span><br><span class="line">a1.sources.r1.channels = c1</span><br></pre></td></tr></table></figure>

<p><strong>(4) hadoop103创建配置文件a1.conf</strong></p>
<p><strong>hadoop103</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">cd</span> /opt/module/flume/job/interceptor</span><br><span class="line">vi a1.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a1.sources = r1</span><br><span class="line">a1.sinks = k1</span><br><span class="line">a1.channels = c1</span><br><span class="line"></span><br><span class="line">a1.sources.r1.type = avro</span><br><span class="line">a1.sources.r1.bind = hadoop103</span><br><span class="line">a1.sources.r1.port = 4242</span><br><span class="line"></span><br><span class="line">a1.sinks.k1.type = logger</span><br><span class="line"></span><br><span class="line">a1.channels.c1.type = memory</span><br><span class="line">a1.channels.c1.capacity = 1000</span><br><span class="line">a1.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line">a1.sinks.k1.channel = c1</span><br><span class="line">a1.sources.r1.channels = c1</span><br></pre></td></tr></table></figure>

<p><strong>(5) 分别启动flume进程</strong></p>
<p>** hadoop103**</p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent -n a1 -c conf/ -f job/interceptor/a1.conf -Dflume.root.logger=INFO,console</span><br></pre></td></tr></table></figure>

<p><strong>hadoop102</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent -n a1 -c conf/ -f job/interceptor/a1.conf -Dflume.root.logger=INFO,console</span><br></pre></td></tr></table></figure>

<p><strong>hadoop101</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent -n a1 -c conf/ -f job/interceptor/a1.conf -Dflume.root.logger=INFO,console</span><br></pre></td></tr></table></figure>

<p><strong>(6) 开启另一个终端，不断发送消息</strong></p>
<p><strong>hadoop101</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">nc hadoop102 44444</span><br><span class="line">aaa</span><br><span class="line">111</span><br><span class="line">1ss</span><br><span class="line">s11</span><br></pre></td></tr></table></figure>

<h2 id="5-6-自定义Source案例"><a href="#5-6-自定义Source案例" class="headerlink" title="5.6 自定义Source案例"></a>5.6 自定义Source案例</h2><p><strong>(1) 实现一个Source类</strong></p>
<figure class="highlight java"><table><tr><td class="code"><pre><span class="line"><span class="keyword">package</span> com.lytdevflume.source;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> org.apache.flume.Context;</span><br><span class="line"><span class="keyword">import</span> org.apache.flume.Event;</span><br><span class="line"><span class="keyword">import</span> org.apache.flume.EventDeliveryException;</span><br><span class="line"><span class="keyword">import</span> org.apache.flume.PollableSource;</span><br><span class="line"><span class="keyword">import</span> org.apache.flume.conf.Configurable;</span><br><span class="line"><span class="keyword">import</span> org.apache.flume.event.SimpleEvent;</span><br><span class="line"><span class="keyword">import</span> org.apache.flume.source.AbstractSource;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> java.util.HashMap;</span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">MySource</span> <span class="keyword">extends</span> <span class="title">AbstractSource</span> <span class="keyword">implements</span> <span class="title">Configurable</span>, <span class="title">PollableSource</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">private</span> String prefix;</span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">long</span> interval;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> Status <span class="title">process</span><span class="params">()</span> <span class="keyword">throws</span> EventDeliveryException </span>&#123;</span><br><span class="line">        Status status = <span class="keyword">null</span>;</span><br><span class="line">        <span class="keyword">try</span> &#123;</span><br><span class="line">            <span class="keyword">for</span> (<span class="keyword">int</span> i = <span class="number">1</span>; i &lt;= <span class="number">5</span>; i++) &#123;</span><br><span class="line">                Event e = <span class="keyword">new</span> SimpleEvent();</span><br><span class="line">                e.setHeaders(<span class="keyword">new</span> HashMap&lt;String, String&gt;());</span><br><span class="line">                e.setBody((prefix + i).getBytes());</span><br><span class="line">                getChannelProcessor().processEvent(e);</span><br><span class="line">                Thread.sleep(interval);</span><br><span class="line">            &#125;</span><br><span class="line">            status = Status.READY;</span><br><span class="line">        &#125; <span class="keyword">catch</span> (InterruptedException e) &#123;</span><br><span class="line">            status = Status.BACKOFF;</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> status;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">long</span> <span class="title">getBackOffSleepIncrement</span><span class="params">()</span> </span>&#123;</span><br><span class="line">        <span class="keyword">return</span> <span class="number">2000</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">long</span> <span class="title">getMaxBackOffSleepInterval</span><span class="params">()</span> </span>&#123;</span><br><span class="line">        <span class="keyword">return</span> <span class="number">20000</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">configure</span><span class="params">(Context context)</span> </span>&#123;</span><br><span class="line">        prefix = context.getString(<span class="string">&quot;source.prefix&quot;</span>,<span class="string">&quot;Log&quot;</span>);</span><br><span class="line">        interval = context.getLong(<span class="string">&quot;source.interval&quot;</span>,<span class="number">1000L</span>);</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<p><strong>(2) hadoop101创建配置文件a1.conf</strong></p>
<p><strong>hadoop101</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">cd</span> /opt/module/flume/job/<span class="built_in">source</span></span><br><span class="line">vi a1.conf</span><br><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a1.sources = r1</span><br><span class="line">a1.sinks = k1</span><br><span class="line">a1.channels = c1</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a1.sources.r1.type = com.lytdevflume.source.MySource</span><br><span class="line">a1.sources.r1.source.prefix= Log</span><br><span class="line">a1.sources.r1.source.interval= 1000</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a1.sinks.k1.type = logger</span><br><span class="line"></span><br><span class="line"><span class="comment"># Use a channel which buffers events in memory</span></span><br><span class="line">a1.channels.c1.type = memory</span><br><span class="line">a1.channels.c1.capacity = 1000</span><br><span class="line">a1.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a1.sources.r1.channels = c1</span><br><span class="line">a1.sinks.k1.channel = c1</span><br></pre></td></tr></table></figure>

<p><strong>(3) 启动flume进程</strong></p>
<p><strong>hadoop101</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent -n a1 -c conf/ -f job/<span class="built_in">source</span>/a1.conf -Dflume.root.logger=INFO,console</span><br></pre></td></tr></table></figure>

<h2 id="5-7-自定义文件Source案例二"><a href="#5-7-自定义文件Source案例二" class="headerlink" title="5.7 自定义文件Source案例二"></a>5.7 自定义文件Source案例二</h2><p><strong>(1) 实现一个Source类，接收MQTT服务器</strong></p>
<p>引入org.eclipse.paho.client.mqttv3.* 类包<br>    自定义一个Source类<br>    自定义一个SimpleMqttClient类<br>    该类用于配置MQTT服务器的链接（IP地址、端口号、用户名、密码、主题）、订阅主题消息、订阅消息的回调<br>    接收到回调函数的消息，进行event的封装，然后发送</p>
<p><strong>(2) 实现一个Source类，接收传感器串口数据</strong></p>
<p>自定义一个Source类<br>    自定义一个串口类<br>    该类用于配置串口类的链接（串口号、波特率、校验位、数据位、停止位）、数据的回调（数据都是以十六进制进行传递）<br>    接收到回调函数的消息，进行event的封装，然后发送</p>
<h2 id="5-8-自定义Sink案例"><a href="#5-8-自定义Sink案例" class="headerlink" title="5.8 自定义Sink案例"></a>5.8 自定义Sink案例</h2><p><strong>(1) 实现一个Sink类</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">package com.lytdevflume.sink;</span><br><span class="line"></span><br><span class="line">import org.apache.flume.*;</span><br><span class="line">import org.apache.flume.conf.Configurable;</span><br><span class="line">import org.apache.flume.sink.AbstractSink;</span><br><span class="line"></span><br><span class="line">public class MySink extends AbstractSink implements Configurable &#123;</span><br><span class="line">    private long interval;</span><br><span class="line">    private String prefix;</span><br><span class="line">    private String suffix;</span><br><span class="line"></span><br><span class="line">    public Status process() throws EventDeliveryException &#123;</span><br><span class="line">        Status status = null;</span><br><span class="line">        Channel channel = this.getChannel();</span><br><span class="line">        Transaction transaction = channel.getTransaction();</span><br><span class="line">        transaction.begin();</span><br><span class="line">        try &#123;</span><br><span class="line">            Event event = null;</span><br><span class="line">            <span class="keyword">while</span> ((event = channel.take()) == null) &#123;</span><br><span class="line">                Thread.sleep(interval);</span><br><span class="line">            &#125;</span><br><span class="line">            byte[] body = event.getBody();</span><br><span class="line">            String line = new String(body, <span class="string">&quot;UTF-8&quot;</span>);</span><br><span class="line">            System.out.println(prefix + line + suffix);</span><br><span class="line">            status = Status.READY;</span><br><span class="line">            transaction.commit();</span><br><span class="line">        &#125; catch (Exception e) &#123;</span><br><span class="line">            transaction.rollback();</span><br><span class="line">            status = Status.BACKOFF;</span><br><span class="line">        &#125; finally &#123;</span><br><span class="line">            transaction.close();</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="built_in">return</span> status;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    public void configure(Context context) &#123;</span><br><span class="line">        prefix = context.getString(<span class="string">&quot;source.prefix&quot;</span>, <span class="string">&quot;start:&quot;</span>);</span><br><span class="line">        suffix = context.getString(<span class="string">&quot;source.suffix&quot;</span>, <span class="string">&quot;:end&quot;</span>);</span><br><span class="line">        interval = context.getLong(<span class="string">&quot;source.interval&quot;</span>, 1000L);</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<p><strong>(2) hadoop101创建配置文件a1.conf</strong></p>
<p><strong>hadoop101</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Name the components on this agent</span></span><br><span class="line">a1.sources = r1</span><br><span class="line">a1.sinks = k1</span><br><span class="line">a1.channels = c1</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe/configure the source</span></span><br><span class="line">a1.sources.r1.type = netcat</span><br><span class="line">a1.sources.r1.bind = localhost</span><br><span class="line">a1.sources.r1.port = 44444</span><br><span class="line"></span><br><span class="line"><span class="comment"># Describe the sink</span></span><br><span class="line">a1.sinks.k1.type = com.lytdevflume.sink.MySink</span><br><span class="line">a1.sinks.k1.source.prefix = lytdev:</span><br><span class="line">a1.sinks.k1.source.suffix = :lytdev</span><br><span class="line">a1.sinks.k1.source.interval = 1000</span><br><span class="line"></span><br><span class="line"><span class="comment"># Use a channel which buffers events in memory</span></span><br><span class="line">a1.channels.c1.type = memory</span><br><span class="line">a1.channels.c1.capacity = 1000</span><br><span class="line">a1.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># Bind the source and sink to the channel</span></span><br><span class="line">a1.sources.r1.channels = c1</span><br><span class="line">a1.sinks.k1.channel = c1</span><br></pre></td></tr></table></figure>

<p><strong>(3) 启动flume进程</strong></p>
<p><strong>hadoop101</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent -n a1 -c conf/ -f job/sink/a1.conf -Dflume.root.logger=INFO,console</span><br></pre></td></tr></table></figure>

<h1 id="六、Flume数据流监控"><a href="#六、Flume数据流监控" class="headerlink" title="六、Flume数据流监控"></a>六、Flume数据流监控</h1><h2 id="6-1-Flume自带监控工具"><a href="#6-1-Flume自带监控工具" class="headerlink" title="6.1 Flume自带监控工具"></a>6.1 Flume自带监控工具</h2><p>执行如下的命令，并访问 <a target="_blank" rel="noopener external nofollow noreferrer" href="http://ip:1234/">http://IP:1234</a></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent -n a1 -c conf/ -f job/sink/a1.conf -Dflume.monitoring.type=http -Dflume.monitoring.port=1234</span><br></pre></td></tr></table></figure>

<h2 id="6-2-Ganglia"><a href="#6-2-Ganglia" class="headerlink" title="6.2 Ganglia"></a>6.2 Ganglia</h2><p>Ganglia由gmond、gmetad和gweb三部分组成</p>
<p><strong>gmond（Ganglia Monitoring Daemon）</strong><br>    gmond是一种轻量级服务，安装在每台需要收集指标数据的节点主机上。使用gmond，你可以很容易收集很多系统指标数据，如CPU、内存、磁盘、网络和活跃进程的数据等</p>
<p><strong>gmetad（Ganglia Meta Daemon）</strong><br>    gmetad整合所有信息，并将其以RRD格式存储至磁盘的服务</p>
<p><strong>gweb（Ganglia Web）</strong><br>    Ganglia可视化工具，gweb是一种利用浏览器显示gmetad所存储数据的PHP前端。在Web界面中以图表方式展现集群的运行状态下收集的多种不同指标数据</p>
<h3 id="6-3-Ganglia监控视图"><a href="#6-3-Ganglia监控视图" class="headerlink" title="6.3 Ganglia监控视图"></a>6.3 Ganglia监控视图</h3><table>
<thead>
<tr>
<th>监控视图名称</th>
<th>描述</th>
</tr>
</thead>
<tbody><tr>
<td>EventPutAttemptCount</td>
<td>source尝试写入channel的事件总数量</td>
</tr>
<tr>
<td>EventPutSuccessCount</td>
<td>成功写入channel且提交的事件总数量</td>
</tr>
<tr>
<td>EventTakeAttemptCount</td>
<td>sink尝试从channel拉取事件的总数量</td>
</tr>
<tr>
<td>EventTakeSuccessCount</td>
<td>sink成功读取的事件的总数量</td>
</tr>
<tr>
<td>StartTime</td>
<td>channel启动的时间（毫秒）</td>
</tr>
<tr>
<td>StopTime</td>
<td>channel停止的时间（毫秒）</td>
</tr>
<tr>
<td>ChannelSize</td>
<td>目前channel中事件的总数量</td>
</tr>
<tr>
<td>ChannelFillPercentage</td>
<td>channel占用百分比</td>
</tr>
<tr>
<td>ChannelCapacity</td>
<td>channel的容量</td>
</tr>
</tbody></table>
<p>EventPutSuccessCount = EventTakeSuccessCount + ChannelSize<br>否则，丢数据。</p>
</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="mailto:undefined" rel="external nofollow noreferrer">liuyuantao</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="https://liuyuantao.gitee.io/2021/09/12/a99eab3d4a13.html">https://liuyuantao.gitee.io/2021/09/12/a99eab3d4a13.html</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="external nofollow noreferrer" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="https://liuyuantao.gitee.io" target="_blank">全栈进阶那些事</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/tags/Hadoop/">Hadoop</a><a class="post-meta__tags" href="/tags/Flume/">Flume</a></div><div class="post_share"><div class="social-share" data-image="https://oscimg.oschina.net/oscnet/20200617165336828.png" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/social-share.js/dist/css/share.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/social-share.js/dist/js/social-share.min.js" defer></script></div></div><div class="post-reward"><div class="reward-button button--animated"><i class="fas fa-qrcode"></i> 打赏</div><div class="reward-main"><ul class="reward-all"><li class="reward-item"><a href="/img/qr_code_wechat.jpg" target="_blank"><img class="post-qr-code-img" src="/img/qr_code_wechat.jpg" alt="微信"/></a><div class="post-qr-code-desc">微信</div></li><li class="reward-item"><a href="/img/qr_code_alipay.jpg" target="_blank"><img class="post-qr-code-img" src="/img/qr_code_alipay.jpg" alt="支付宝"/></a><div class="post-qr-code-desc">支付宝</div></li></ul></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/2021/10/12/245dd8355f34.html"><img class="prev-cover" src="https://img-blog.csdnimg.cn/20200620092054385.png" onerror="onerror=null;src='/img/404.jpg'" alt="cover of previous post"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">一文彻底搞懂Kafka</div></div></a></div><div class="next-post pull-right"><a href="/2021/07/12/1e41bdf3e092.html"><img class="next-cover" src="https://img-blog.csdnimg.cn/20200518143448459.png" onerror="onerror=null;src='/img/404.jpg'" alt="cover of next post"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">一文彻底搞懂Zookeeper</div></div></a></div></nav><div class="relatedPosts"><div class="headline"><i class="fas fa-thumbs-up fa-fw"></i><span>相关推荐</span></div><div class="relatedPosts-list"><div><a href="/2020/11/07/7d14a19c4606.html" title="DataNode工作机制"><img class="cover" src="https://oscimg.oschina.net/oscnet/up-0a831a2854d1859bc355a98a741ee74452b.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-11-07</div><div class="title">DataNode工作机制</div></div></a></div><div><a href="/2020/11/07/9cc4868ea813.html" title="Hadoop-Yarn常用的调优参数"><img class="cover" src="https://cdn.pixabay.com/photo/2021/08/08/10/34/ocean-6530523__340.jpg" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-11-07</div><div class="title">Hadoop-Yarn常用的调优参数</div></div></a></div><div><a href="/2020/11/07/5f9b1cc664c1.html" title="HDFS—集群扩容及缩容"><img class="cover" src="https://oscimg.oschina.net/oscnet/up-acb08d98e431429592cdb8802409bb29a19.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-11-07</div><div class="title">HDFS—集群扩容及缩容</div></div></a></div><div><a href="/2020/11/07/a8fda6be4c8d.html" title="HDFS—存储优化（纠删码）"><img class="cover" src="https://oscimg.oschina.net/oscnet/up-87ba27811580330155a29254c76223dd59f.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-11-07</div><div class="title">HDFS—存储优化（纠删码）</div></div></a></div><div><a href="/2020/11/07/3a181e232503.html" title="Hadoop小文件优化方法"><img class="cover" src="https://oscimg.oschina.net/oscnet/up-1681b15baa2d4bc66ceb813b9d6fcd4ad55.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-11-07</div><div class="title">Hadoop小文件优化方法</div></div></a></div><div><a href="/2020/11/07/fd53f751c2eb.html" title="Hadoop3.X分布式运行环境搭建手记"><img class="cover" src="https://oscimg.oschina.net/oscnet/c646ab2c9196c3eacc49b81b10d47f7a191.jpg" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2020-11-07</div><div class="title">Hadoop3.X分布式运行环境搭建手记</div></div></a></div></div></div></div><div class="aside-content" id="aside-content"><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%B8%80%E3%80%81Flume%E7%9A%84%E7%AE%80%E4%BB%8B"><span class="toc-text">一、Flume的简介</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#1-1-Flume%E5%9F%BA%E6%9C%AC%E6%A6%82%E5%BF%B5"><span class="toc-text">1.1 Flume基本概念</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#1-2-Flume%E5%9F%BA%E6%9C%AC%E7%BB%84%E4%BB%B6"><span class="toc-text">1.2 Flume基本组件</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%BA%8C%E3%80%81Flume%E7%9A%84%E5%AE%89%E8%A3%85%E5%92%8C%E5%85%A5%E9%97%A8%E6%A1%88%E4%BE%8B"><span class="toc-text">二、Flume的安装和入门案例</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#2-1-Flume%E5%AE%89%E8%A3%85"><span class="toc-text">2.1 Flume安装</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-2-Flume%E6%A1%88%E4%BE%8B-%E7%9B%91%E5%90%AC%E6%95%B0%E6%8D%AE%E7%AB%AF%E5%8F%A3"><span class="toc-text">2.2 Flume案例-监听数据端口</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-3-Flume%E6%A1%88%E4%BE%8B-%E5%AE%9E%E6%97%B6%E7%9B%91%E6%8E%A7%E5%8D%95%E4%B8%AA%E8%BF%BD%E5%8A%A0%E6%96%87%E4%BB%B6"><span class="toc-text">2.3 Flume案例-实时监控单个追加文件</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-5-Flume%E6%A1%88%E4%BE%8B-%E5%AE%9E%E6%97%B6%E7%9B%91%E6%8E%A7%E7%9B%AE%E5%BD%95%E4%B8%8B%E7%9A%84%E5%A4%9A%E4%B8%AA%E8%BF%BD%E5%8A%A0%E6%96%87%E4%BB%B6"><span class="toc-text">2.5 Flume案例-实时监控目录下的多个追加文件</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%B8%89%E3%80%81Flume%E7%9A%84%E8%BF%9B%E9%98%B6"><span class="toc-text">三、Flume的进阶</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#3-1-Flume%E4%BA%8B%E5%8A%A1"><span class="toc-text">3.1 Flume事务</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#3-2-Flume-Agent%E5%86%85%E9%83%A8%E5%8E%9F%E7%90%86"><span class="toc-text">3.2 Flume Agent内部原理</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E5%9B%9B%E3%80%81Flume%E7%9A%84%E6%8B%93%E6%89%91%E7%BB%93%E6%9E%84"><span class="toc-text">四、Flume的拓扑结构</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#4-2-%E5%A4%8D%E5%88%B6%E5%92%8C%E5%A4%9A%E8%B7%AF%E5%A4%8D%E7%94%A8"><span class="toc-text">4.2 复制和多路复用</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#4-3-%E8%B4%9F%E8%BD%BD%E5%9D%87%E8%A1%A1%E5%92%8C%E6%95%85%E9%9A%9C%E8%BD%AC%E7%A7%BB"><span class="toc-text">4.3 负载均衡和故障转移</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#4-4-%E8%81%9A%E5%90%88"><span class="toc-text">4.4 聚合</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%BA%94%E3%80%81Flume%E7%9A%84%E4%BC%81%E4%B8%9A%E5%BC%80%E5%8F%91%E5%AE%9E%E4%BE%8B"><span class="toc-text">五、Flume的企业开发实例</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#5-1-%E5%A4%8D%E5%88%B6%E5%92%8C%E5%A4%9A%E8%B7%AF%E5%A4%8D%E7%94%A8"><span class="toc-text">5.1 复制和多路复用</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-2-%E6%95%85%E9%9A%9C%E8%BD%AC%E7%A7%BB"><span class="toc-text">5.2 故障转移</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-3-%E8%B4%9F%E8%BD%BD%E5%9D%87%E8%A1%A1"><span class="toc-text">5.3 负载均衡</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-4-%E8%81%9A%E5%90%88"><span class="toc-text">5.4 聚合</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-5-%E8%87%AA%E5%AE%9A%E4%B9%89Interceptor%E6%A1%88%E4%BE%8B"><span class="toc-text">5.5 自定义Interceptor案例</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-6-%E8%87%AA%E5%AE%9A%E4%B9%89Source%E6%A1%88%E4%BE%8B"><span class="toc-text">5.6 自定义Source案例</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-7-%E8%87%AA%E5%AE%9A%E4%B9%89%E6%96%87%E4%BB%B6Source%E6%A1%88%E4%BE%8B%E4%BA%8C"><span class="toc-text">5.7 自定义文件Source案例二</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-8-%E8%87%AA%E5%AE%9A%E4%B9%89Sink%E6%A1%88%E4%BE%8B"><span class="toc-text">5.8 自定义Sink案例</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E5%85%AD%E3%80%81Flume%E6%95%B0%E6%8D%AE%E6%B5%81%E7%9B%91%E6%8E%A7"><span class="toc-text">六、Flume数据流监控</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#6-1-Flume%E8%87%AA%E5%B8%A6%E7%9B%91%E6%8E%A7%E5%B7%A5%E5%85%B7"><span class="toc-text">6.1 Flume自带监控工具</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#6-2-Ganglia"><span class="toc-text">6.2 Ganglia</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#6-3-Ganglia%E7%9B%91%E6%8E%A7%E8%A7%86%E5%9B%BE"><span class="toc-text">6.3 Ganglia监控视图</span></a></li></ol></li></ol></li></ol></div></div></div></div></main><footer id="footer"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2021 By liuyuantao</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener external nofollow noreferrer" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener external nofollow noreferrer" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div><div class="footer_custom_text">扫描二维码,关注我们!了解更多全栈技能!<br/><img style="width:100px;height:100px;" src="/img/qrcode_for_gh_da91cebd00cc_258.jpg" ></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="font-plus" type="button" title="放大字体"><i class="fas fa-plus"></i></button><button id="font-minus" type="button" title="缩小字体"><i class="fas fa-minus"></i></button><button id="translateLink" type="button" title="简繁转换">简</button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="local-search"><div class="search-dialog"><div class="search-dialog__title" id="local-search-title">本地搜索</div><div id="local-input-panel"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div></div><hr/><div id="local-search-results"></div><span class="search-close-button"><i class="fas fa-times"></i></span></div><div id="search-mask"></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="/js/tw_cn.js"></script><script src="https://cdn.jsdelivr.net/npm/medium-zoom/dist/medium-zoom.min.js"></script><script src="https://cdn.jsdelivr.net/npm/instant.page/instantpage.min.js" type="module"></script><script src="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.js"></script><script src="/js/search/local-search.js"></script><script>var preloader = {
  endLoading: () => {
    document.body.style.overflow = 'auto';
    document.getElementById('loading-box').classList.add("loaded")
  },
  initLoading: () => {
    document.body.style.overflow = '';
    document.getElementById('loading-box').classList.remove("loaded")

  }
}
window.addEventListener('load',preloader.endLoading())</script><div class="js-pjax"></div><script defer="defer" id="fluttering_ribbon" mobile="false" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/canvas-fluttering-ribbon.min.js"></script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>